/**
 * 2017年5月17日
 */
package cn.edu.bjtu.tokenization;

import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor;

import cn.edu.bjtu.core.LoggerSupport;

/**
 * 分词前预处理
 * 不想保留的话直接返回""
 * @author Alex
 *
 */
public class AnsjTokenPreprocessor extends LoggerSupport implements TokenPreProcess   {
	
	CommonPreprocessor cp = new CommonPreprocessor(); 
	@Override
	public String preProcess(String token) {
		String res = cp.preProcess(token);
		return res;
	}

}
